Overview of tutorial

This tutorial simulates a population effect size of Cohen’s d = 0.5 for different sample sizes, and examines the relationship between Cohen’s d, its 95% Confidence Interval, and the significance of the t-test’s p-value.

By the end of this lesson you should understand that p-values are re-expressions of the same information conveyed by Confidence Intervals, and that statistical power is a re-expression of the width of Confidence Intervals.

Citation & License

Citation:

Ian Hussey (2024) Improving your statistical inferences through simulation studies in R. https://github.com/ianhussey/simulation-course

License:

CC BY 4.0

Dependencies

library(tidyr)
library(dplyr)
library(purrr) 
library(stringr)
library(forcats)
library(ggplot2)
library(scales)
library(patchwork)
library(knitr)
library(kableExtra)
library(janitor)
library(effsize)

Simulation

# functions for simulation
generate_data <- function(n_per_condition,
                          mean_control,
                          mean_intervention,
                          sd) {
  
  data_control <- 
    tibble(condition = "control",
           score = rnorm(n = n_per_condition, mean = mean_control, sd = sd))
  
  data_intervention <- 
    tibble(condition = "intervention",
           score = rnorm(n = n_per_condition, mean = mean_intervention, sd = sd))
  
  data_combined <- bind_rows(data_control,
                             data_intervention) |>
    mutate(condition = fct_relevel(condition, "intervention", "control"))
  
  return(data_combined)
}

analyze <- function(data) {

  res_t_test <- t.test(formula = score ~ condition, 
                       data = data,
                       var.equal = TRUE,
                       alternative = "two.sided")
  
  res_cohens_d <- effsize::cohen.d(formula = score ~ condition,
                                   data = data,
                                   pooled = TRUE)
  
  res <- tibble(p = res_t_test$p.value, 
                cohens_d = res_cohens_d$estimate,
                cohens_d_ci_lower = res_cohens_d$conf.int[1],
                cohens_d_ci_upper = res_cohens_d$conf.int[2])

  return(res)
}


# set seed
set.seed(42)

# simulation parameters
experiment_parameters <- expand_grid(
  n_per_condition = seq(from = 10, to = 90, by = 10),
  mean_control = 0,
  mean_intervention = 0.5,
  sd = 1,
  iteration = 1:1000
) 

# run simulation
simulation <- experiment_parameters |>
  mutate(generated_data = pmap(list(n_per_condition, 
                                    mean_control,
                                    mean_intervention,
                                    sd),
                               generate_data)) |>
  mutate(results = pmap(list(generated_data),
                        analyze))

Cohen’s d by sample size

# wrangle
simulation |>
  unnest(results) |>
  # plot
  ggplot(aes(n_per_condition*2, cohens_d)) +
  geom_jitter(alpha = 0.25) +
  geom_hline(yintercept = 0, linetype = "dotted") +
  scale_x_continuous(breaks = breaks_pretty(n = 10),
                     name = "Total N") +
  scale_y_continuous(breaks = breaks_pretty(n = 10),
                     #limits = c(0,1),
                     name = "Cohen's d") +
  theme_linedraw() +
  scale_color_viridis_d(begin = 0.3, end = 0.7) +
  ggtitle("Population Cohen's d = 0.5")

Ordered by statistical significance

Let’s color the Cohen’s ds by their statistical significance.

# wrangle
simulation |>
  unnest(results) |>
  mutate(significant = p < .05) |>
  # plot
  ggplot(aes(n_per_condition*2, cohens_d, color = significant)) +
  geom_jitter(alpha = 0.25) +
  geom_hline(yintercept = 0, linetype = "dotted") +
  scale_x_continuous(breaks = breaks_pretty(n = 10),
                     name = "Total N") +
  scale_y_continuous(breaks = breaks_pretty(n = 10),
                     #limits = c(0,1),
                     name = "Cohen's d") +
  theme_linedraw() +
  scale_color_viridis_d(begin = 0.3, end = 0.7) +
  ggtitle("Population Cohen's d = 0.5")

Cohen’s d and its 95% CIs

Let’s add the 95% Confidence Intervals.

Randomly select one dataset per sample size

# wrangle
simulation |>
  unnest(results) |>
  mutate(significant = p < .05) |>
  group_by(n_per_condition) |>
  slice_sample(n = 1) |>
  ungroup() |>
  # plot
  ggplot(aes(n_per_condition*2, cohens_d, color = significant)) +
  geom_point() +
  geom_linerange(aes(ymin = cohens_d_ci_lower, ymax = cohens_d_ci_upper)) +
  geom_hline(yintercept = 0, linetype = "dotted") +
  scale_x_continuous(breaks = breaks_pretty(n = 10),
                     name = "Total N") +
  scale_y_continuous(breaks = breaks_pretty(n = 10),
                     #limits = c(0,1),
                     name = "Cohen's d") +
  theme_linedraw() +
  scale_color_viridis_d(begin = 0.3, end = 0.7) +
  ggtitle("Population Cohen's d = 0.5\nOne randomly selected dataset per sample size")

All datasets

# wrangle
simulation |>
  unnest(results) |>
  mutate(significant = p < .05,
         total_n = paste("N =", n_per_condition*2),
         total_n = fct_reorder(total_n, as.numeric(str_extract(total_n, "\\d+")))) |>
  # plot
  ggplot(aes(iteration, cohens_d, color = significant)) +
  geom_point(alpha = 0.5) +
  geom_linerange(aes(ymin = cohens_d_ci_lower, ymax = cohens_d_ci_upper), alpha = 0.2) +
  geom_hline(yintercept = 0, linetype = "dotted") +
  scale_x_continuous(breaks = breaks_pretty(n = 10),
                     name = "Iteration") +
  scale_y_continuous(breaks = breaks_pretty(n = 10),
                     #limits = c(0,1),
                     name = "Cohen's d") +
  theme_linedraw() +
  scale_color_viridis_d(begin = 0.3, end = 0.7) +
  facet_wrap(~ total_n, ncol = 3, scales = "free_y")

Ordered by Cohen’s d

The above plot is hard to understand. Let’s order the Cohen’s ds from smallest to largest.

# wrangle
simulation |>
  unnest(results) |>
  mutate(significant = p < .05,
         total_n = paste("N =", n_per_condition*2),
         total_n = fct_reorder(total_n, as.numeric(str_extract(total_n, "\\d+")))) |>
  arrange(n_per_condition, cohens_d) |>
  group_by(n_per_condition) |>
  mutate(rank = row_number()) |>
  ungroup() |>
  # plot
  ggplot(aes(rank, cohens_d, color = significant)) +
  geom_point() +
  geom_linerange(aes(ymin = cohens_d_ci_lower, ymax = cohens_d_ci_upper), alpha = 0.2) +
  geom_hline(yintercept = 0, linetype = "dotted") +
  scale_x_continuous(breaks = breaks_pretty(n = 10),
                     name = "Ranked iteration") +
  scale_y_continuous(breaks = breaks_pretty(n = 10),
                     #limits = c(0,1),
                     name = "Cohen's d") +
  theme_linedraw() +
  scale_color_viridis_d(begin = 0.3, end = 0.7) +
  facet_wrap(~ total_n, ncol = 3)

  • Notice the relationship between 95% CI and p value significance.
  • A certain percentage of Cohen’s ds are green vs. blue. What is this percentage also called? What statistical property?

Mean Cohen’s d by sample size

Now let’s average over the Cohen’s ds in each condition to find the mean Cohen’s d and its mean 95% CIs.

# wrangle
simulation_summary <- simulation |>
  # unnest results
  unnest(results) |>
  group_by(n_per_condition) |>
  summarize(proportion_significant = mean(p < .05), 
            mean_cohens_d = mean(cohens_d),
            mean_cohens_d_ci_lower = mean(cohens_d_ci_lower),
            mean_cohens_d_ci_upper = mean(cohens_d_ci_upper)) |>
  mutate(centered_mean_cohens_d_ci_lower = mean_cohens_d_ci_lower - mean_cohens_d,
         centered_mean_cohens_d_ci_upper = mean_cohens_d_ci_upper - mean_cohens_d)

# plot results
p1 <- ggplot(simulation_summary, aes(n_per_condition*2, mean_cohens_d)) +
  geom_point() +
  geom_linerange(aes(ymin = mean_cohens_d_ci_lower, ymax = mean_cohens_d_ci_upper)) +
  geom_hline(yintercept = 0, linetype = "dotted") +
  scale_x_continuous(breaks = breaks_pretty(n = 10),
                     name = "Total sample size") +
  scale_y_continuous(breaks = breaks_pretty(n = 5),
                     #limits = c(0,1),
                     name = "Mean Cohen's d\n(and mean 95% CIs)") +
  theme_linedraw() +
  ggtitle("Population Cohen's d = 0.5")

p1

Mean Cohen’s d and power by sample size

p2 <- ggplot(simulation_summary, aes(n_per_condition*2, proportion_significant)) +
  geom_point() +
  geom_hline(yintercept = 0.05, linetype = "dotted") +
  geom_hline(yintercept = 0.80, linetype = "dotted") +
  scale_x_continuous(breaks = breaks_pretty(n = 10),
                     name = "Total sample size") +
  scale_y_continuous(breaks = breaks_pretty(n = 5),
                     limits = c(0,1),
                     name = "Proportion of significant\np-values") +
  theme_linedraw() 

p1 + p2 + plot_layout(ncol = 1)

Session info

sessionInfo()
## R version 4.3.3 (2024-02-29)
## Platform: aarch64-apple-darwin20 (64-bit)
## Running under: macOS 15.2
## 
## Matrix products: default
## BLAS:   /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/lib/libRblas.0.dylib 
## LAPACK: /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/lib/libRlapack.dylib;  LAPACK version 3.11.0
## 
## locale:
## [1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8
## 
## time zone: Europe/Zurich
## tzcode source: internal
## 
## attached base packages:
## [1] stats     graphics  grDevices utils     datasets  methods   base     
## 
## other attached packages:
##  [1] effsize_0.8.1        janitor_2.2.1        kableExtra_1.4.0    
##  [4] knitr_1.49           patchwork_1.2.0.9000 scales_1.3.0        
##  [7] ggplot2_3.5.1        forcats_1.0.0        stringr_1.5.1       
## [10] purrr_1.0.4          dplyr_1.1.4          tidyr_1.3.1         
## 
## loaded via a namespace (and not attached):
##  [1] gtable_0.3.6      jsonlite_1.8.9    compiler_4.3.3    tidyselect_1.2.1 
##  [5] xml2_1.3.6        snakecase_0.11.1  jquerylib_0.1.4   systemfonts_1.0.6
##  [9] yaml_2.3.10       fastmap_1.2.0     R6_2.6.1          generics_0.1.3   
## [13] tibble_3.2.1      munsell_0.5.1     lubridate_1.9.4   svglite_2.1.3    
## [17] bslib_0.8.0       pillar_1.10.1     rlang_1.1.5       cachem_1.1.0     
## [21] stringi_1.8.4     xfun_0.49         sass_0.4.9        timechange_0.3.0 
## [25] viridisLite_0.4.2 cli_3.6.4         withr_3.0.2       magrittr_2.0.3   
## [29] digest_0.6.37     grid_4.3.3        rstudioapi_0.17.1 lifecycle_1.0.4  
## [33] vctrs_0.6.5       evaluate_1.0.1    glue_1.8.0        farver_2.1.2     
## [37] colorspace_2.1-1  rmarkdown_2.29    tools_4.3.3       pkgconfig_2.0.3  
## [41] htmltools_0.5.8.1
LS0tCnRpdGxlOiAiVW5kZXJzdGFuZGluZyB0aGUgbGluayBiZXR3ZWVuIHAtdmFsdWVzLCBDb25maWRlbmNlIEludGVydmFscywgYW5kIHBvd2VyIgphdXRob3I6ICJJYW4gSHVzc2V5IgpkYXRlOiAiYHIgZm9ybWF0KFN5cy50aW1lKCksICclZCAlQiwgJVknKWAiCm91dHB1dDoKICBodG1sX2RvY3VtZW50OgogICAgY29kZV9kb3dubG9hZDogdHJ1ZQogICAgY29kZV9mb2xkaW5nOiBzaG93CiAgICBoaWdobGlnaHQ6IGhhZGRvY2sKICAgIHRoZW1lOiBmbGF0bHkKICAgIHRvYzogeWVzCiAgICB0b2NfZmxvYXQ6IHllcwotLS0KCiMgT3ZlcnZpZXcgb2YgdHV0b3JpYWwKClRoaXMgdHV0b3JpYWwgc2ltdWxhdGVzIGEgcG9wdWxhdGlvbiBlZmZlY3Qgc2l6ZSBvZiBDb2hlbidzIGQgPSAwLjUgZm9yIGRpZmZlcmVudCBzYW1wbGUgc2l6ZXMsIGFuZCBleGFtaW5lcyB0aGUgcmVsYXRpb25zaGlwIGJldHdlZW4gQ29oZW4ncyBkLCBpdHMgOTUlIENvbmZpZGVuY2UgSW50ZXJ2YWwsIGFuZCB0aGUgc2lnbmlmaWNhbmNlIG9mIHRoZSB0LXRlc3QncyAqcCotdmFsdWUuIAoKQnkgdGhlIGVuZCBvZiB0aGlzIGxlc3NvbiB5b3Ugc2hvdWxkIHVuZGVyc3RhbmQgdGhhdCAqcCotdmFsdWVzIGFyZSByZS1leHByZXNzaW9ucyBvZiB0aGUgc2FtZSBpbmZvcm1hdGlvbiBjb252ZXllZCBieSBDb25maWRlbmNlIEludGVydmFscywgYW5kIHRoYXQgc3RhdGlzdGljYWwgcG93ZXIgaXMgYSByZS1leHByZXNzaW9uIG9mIHRoZSB3aWR0aCBvZiBDb25maWRlbmNlIEludGVydmFscy4KCiMgQ2l0YXRpb24gJiBMaWNlbnNlCgpDaXRhdGlvbjogCgpJYW4gSHVzc2V5ICgyMDI0KSBJbXByb3ZpbmcgeW91ciBzdGF0aXN0aWNhbCBpbmZlcmVuY2VzIHRocm91Z2ggc2ltdWxhdGlvbiBzdHVkaWVzIGluIFIuIGh0dHBzOi8vZ2l0aHViLmNvbS9pYW5odXNzZXkvc2ltdWxhdGlvbi1jb3Vyc2UKCkxpY2Vuc2U6IAoKW0NDIEJZIDQuMF0oaHR0cHM6Ly9jcmVhdGl2ZWNvbW1vbnMub3JnL2xpY2Vuc2VzL2J5LzQuMC9kZWVkLmVuKQoKYGBge3IsIGluY2x1ZGU9RkFMU0V9CgojIHNldCBkZWZhdWx0IGNodW5rIG9wdGlvbnMKa25pdHI6Om9wdHNfY2h1bmskc2V0KG1lc3NhZ2UgPSBGQUxTRSwKICAgICAgICAgICAgICAgICAgICAgIHdhcm5pbmcgPSBGQUxTRSkKCiMgZGlzYWJsZSBzY2llbnRpZmljIG5vdGF0aW9uCm9wdGlvbnMoc2NpcGVuID0gOTk5KSAKCmBgYAoKIyBEZXBlbmRlbmNpZXMKCmBgYHtyfQoKbGlicmFyeSh0aWR5cikKbGlicmFyeShkcGx5cikKbGlicmFyeShwdXJycikgCmxpYnJhcnkoc3RyaW5ncikKbGlicmFyeShmb3JjYXRzKQpsaWJyYXJ5KGdncGxvdDIpCmxpYnJhcnkoc2NhbGVzKQpsaWJyYXJ5KHBhdGNod29yaykKbGlicmFyeShrbml0cikKbGlicmFyeShrYWJsZUV4dHJhKQpsaWJyYXJ5KGphbml0b3IpCmxpYnJhcnkoZWZmc2l6ZSkKCmBgYAoKIyBTaW11bGF0aW9uCgpgYGB7cn0KCiMgZnVuY3Rpb25zIGZvciBzaW11bGF0aW9uCmdlbmVyYXRlX2RhdGEgPC0gZnVuY3Rpb24obl9wZXJfY29uZGl0aW9uLAogICAgICAgICAgICAgICAgICAgICAgICAgIG1lYW5fY29udHJvbCwKICAgICAgICAgICAgICAgICAgICAgICAgICBtZWFuX2ludGVydmVudGlvbiwKICAgICAgICAgICAgICAgICAgICAgICAgICBzZCkgewogIAogIGRhdGFfY29udHJvbCA8LSAKICAgIHRpYmJsZShjb25kaXRpb24gPSAiY29udHJvbCIsCiAgICAgICAgICAgc2NvcmUgPSBybm9ybShuID0gbl9wZXJfY29uZGl0aW9uLCBtZWFuID0gbWVhbl9jb250cm9sLCBzZCA9IHNkKSkKICAKICBkYXRhX2ludGVydmVudGlvbiA8LSAKICAgIHRpYmJsZShjb25kaXRpb24gPSAiaW50ZXJ2ZW50aW9uIiwKICAgICAgICAgICBzY29yZSA9IHJub3JtKG4gPSBuX3Blcl9jb25kaXRpb24sIG1lYW4gPSBtZWFuX2ludGVydmVudGlvbiwgc2QgPSBzZCkpCiAgCiAgZGF0YV9jb21iaW5lZCA8LSBiaW5kX3Jvd3MoZGF0YV9jb250cm9sLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRhdGFfaW50ZXJ2ZW50aW9uKSB8PgogICAgbXV0YXRlKGNvbmRpdGlvbiA9IGZjdF9yZWxldmVsKGNvbmRpdGlvbiwgImludGVydmVudGlvbiIsICJjb250cm9sIikpCiAgCiAgcmV0dXJuKGRhdGFfY29tYmluZWQpCn0KCmFuYWx5emUgPC0gZnVuY3Rpb24oZGF0YSkgewoKICByZXNfdF90ZXN0IDwtIHQudGVzdChmb3JtdWxhID0gc2NvcmUgfiBjb25kaXRpb24sIAogICAgICAgICAgICAgICAgICAgICAgIGRhdGEgPSBkYXRhLAogICAgICAgICAgICAgICAgICAgICAgIHZhci5lcXVhbCA9IFRSVUUsCiAgICAgICAgICAgICAgICAgICAgICAgYWx0ZXJuYXRpdmUgPSAidHdvLnNpZGVkIikKICAKICByZXNfY29oZW5zX2QgPC0gZWZmc2l6ZTo6Y29oZW4uZChmb3JtdWxhID0gc2NvcmUgfiBjb25kaXRpb24sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGF0YSA9IGRhdGEsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcG9vbGVkID0gVFJVRSkKICAKICByZXMgPC0gdGliYmxlKHAgPSByZXNfdF90ZXN0JHAudmFsdWUsIAogICAgICAgICAgICAgICAgY29oZW5zX2QgPSByZXNfY29oZW5zX2QkZXN0aW1hdGUsCiAgICAgICAgICAgICAgICBjb2hlbnNfZF9jaV9sb3dlciA9IHJlc19jb2hlbnNfZCRjb25mLmludFsxXSwKICAgICAgICAgICAgICAgIGNvaGVuc19kX2NpX3VwcGVyID0gcmVzX2NvaGVuc19kJGNvbmYuaW50WzJdKQoKICByZXR1cm4ocmVzKQp9CgoKIyBzZXQgc2VlZApzZXQuc2VlZCg0MikKCiMgc2ltdWxhdGlvbiBwYXJhbWV0ZXJzCmV4cGVyaW1lbnRfcGFyYW1ldGVycyA8LSBleHBhbmRfZ3JpZCgKICBuX3Blcl9jb25kaXRpb24gPSBzZXEoZnJvbSA9IDEwLCB0byA9IDkwLCBieSA9IDEwKSwKICBtZWFuX2NvbnRyb2wgPSAwLAogIG1lYW5faW50ZXJ2ZW50aW9uID0gMC41LAogIHNkID0gMSwKICBpdGVyYXRpb24gPSAxOjEwMDAKKSAKCiMgcnVuIHNpbXVsYXRpb24Kc2ltdWxhdGlvbiA8LSBleHBlcmltZW50X3BhcmFtZXRlcnMgfD4KICBtdXRhdGUoZ2VuZXJhdGVkX2RhdGEgPSBwbWFwKGxpc3Qobl9wZXJfY29uZGl0aW9uLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWVhbl9jb250cm9sLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtZWFuX2ludGVydmVudGlvbiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc2QpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZ2VuZXJhdGVfZGF0YSkpIHw+CiAgbXV0YXRlKHJlc3VsdHMgPSBwbWFwKGxpc3QoZ2VuZXJhdGVkX2RhdGEpLAogICAgICAgICAgICAgICAgICAgICAgICBhbmFseXplKSkKCmBgYAoKIyBDb2hlbidzIGQgYnkgc2FtcGxlIHNpemUKCmBgYHtyIGZpZy5oZWlnaHQ9NSwgZmlnLndpZHRoPTZ9CgojIHdyYW5nbGUKc2ltdWxhdGlvbiB8PgogIHVubmVzdChyZXN1bHRzKSB8PgogICMgcGxvdAogIGdncGxvdChhZXMobl9wZXJfY29uZGl0aW9uKjIsIGNvaGVuc19kKSkgKwogIGdlb21faml0dGVyKGFscGhhID0gMC4yNSkgKwogIGdlb21faGxpbmUoeWludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gImRvdHRlZCIpICsKICBzY2FsZV94X2NvbnRpbnVvdXMoYnJlYWtzID0gYnJlYWtzX3ByZXR0eShuID0gMTApLAogICAgICAgICAgICAgICAgICAgICBuYW1lID0gIlRvdGFsIE4iKSArCiAgc2NhbGVfeV9jb250aW51b3VzKGJyZWFrcyA9IGJyZWFrc19wcmV0dHkobiA9IDEwKSwKICAgICAgICAgICAgICAgICAgICAgI2xpbWl0cyA9IGMoMCwxKSwKICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJDb2hlbidzIGQiKSArCiAgdGhlbWVfbGluZWRyYXcoKSArCiAgc2NhbGVfY29sb3JfdmlyaWRpc19kKGJlZ2luID0gMC4zLCBlbmQgPSAwLjcpICsKICBnZ3RpdGxlKCJQb3B1bGF0aW9uIENvaGVuJ3MgZCA9IDAuNSIpCgpgYGAKCiMjIE9yZGVyZWQgYnkgc3RhdGlzdGljYWwgc2lnbmlmaWNhbmNlCgpMZXQncyBjb2xvciB0aGUgQ29oZW4ncyBkcyBieSB0aGVpciBzdGF0aXN0aWNhbCBzaWduaWZpY2FuY2UuIAoKYGBge3IgZmlnLmhlaWdodD01LCBmaWcud2lkdGg9Nn0KCiMgd3JhbmdsZQpzaW11bGF0aW9uIHw+CiAgdW5uZXN0KHJlc3VsdHMpIHw+CiAgbXV0YXRlKHNpZ25pZmljYW50ID0gcCA8IC4wNSkgfD4KICAjIHBsb3QKICBnZ3Bsb3QoYWVzKG5fcGVyX2NvbmRpdGlvbioyLCBjb2hlbnNfZCwgY29sb3IgPSBzaWduaWZpY2FudCkpICsKICBnZW9tX2ppdHRlcihhbHBoYSA9IDAuMjUpICsKICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQgPSAwLCBsaW5ldHlwZSA9ICJkb3R0ZWQiKSArCiAgc2NhbGVfeF9jb250aW51b3VzKGJyZWFrcyA9IGJyZWFrc19wcmV0dHkobiA9IDEwKSwKICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJUb3RhbCBOIikgKwogIHNjYWxlX3lfY29udGludW91cyhicmVha3MgPSBicmVha3NfcHJldHR5KG4gPSAxMCksCiAgICAgICAgICAgICAgICAgICAgICNsaW1pdHMgPSBjKDAsMSksCiAgICAgICAgICAgICAgICAgICAgIG5hbWUgPSAiQ29oZW4ncyBkIikgKwogIHRoZW1lX2xpbmVkcmF3KCkgKwogIHNjYWxlX2NvbG9yX3ZpcmlkaXNfZChiZWdpbiA9IDAuMywgZW5kID0gMC43KSArCiAgZ2d0aXRsZSgiUG9wdWxhdGlvbiBDb2hlbidzIGQgPSAwLjUiKQoKYGBgCgojIENvaGVuJ3MgZCBhbmQgaXRzIDk1JSBDSXMgCgpMZXQncyBhZGQgdGhlIDk1JSBDb25maWRlbmNlIEludGVydmFscy4KCiMjIFJhbmRvbWx5IHNlbGVjdCBvbmUgZGF0YXNldCBwZXIgc2FtcGxlIHNpemUKCmBgYHtyfQoKIyB3cmFuZ2xlCnNpbXVsYXRpb24gfD4KICB1bm5lc3QocmVzdWx0cykgfD4KICBtdXRhdGUoc2lnbmlmaWNhbnQgPSBwIDwgLjA1KSB8PgogIGdyb3VwX2J5KG5fcGVyX2NvbmRpdGlvbikgfD4KICBzbGljZV9zYW1wbGUobiA9IDEpIHw+CiAgdW5ncm91cCgpIHw+CiAgIyBwbG90CiAgZ2dwbG90KGFlcyhuX3Blcl9jb25kaXRpb24qMiwgY29oZW5zX2QsIGNvbG9yID0gc2lnbmlmaWNhbnQpKSArCiAgZ2VvbV9wb2ludCgpICsKICBnZW9tX2xpbmVyYW5nZShhZXMoeW1pbiA9IGNvaGVuc19kX2NpX2xvd2VyLCB5bWF4ID0gY29oZW5zX2RfY2lfdXBwZXIpKSArCiAgZ2VvbV9obGluZSh5aW50ZXJjZXB0ID0gMCwgbGluZXR5cGUgPSAiZG90dGVkIikgKwogIHNjYWxlX3hfY29udGludW91cyhicmVha3MgPSBicmVha3NfcHJldHR5KG4gPSAxMCksCiAgICAgICAgICAgICAgICAgICAgIG5hbWUgPSAiVG90YWwgTiIpICsKICBzY2FsZV95X2NvbnRpbnVvdXMoYnJlYWtzID0gYnJlYWtzX3ByZXR0eShuID0gMTApLAogICAgICAgICAgICAgICAgICAgICAjbGltaXRzID0gYygwLDEpLAogICAgICAgICAgICAgICAgICAgICBuYW1lID0gIkNvaGVuJ3MgZCIpICsKICB0aGVtZV9saW5lZHJhdygpICsKICBzY2FsZV9jb2xvcl92aXJpZGlzX2QoYmVnaW4gPSAwLjMsIGVuZCA9IDAuNykgKwogIGdndGl0bGUoIlBvcHVsYXRpb24gQ29oZW4ncyBkID0gMC41XG5PbmUgcmFuZG9tbHkgc2VsZWN0ZWQgZGF0YXNldCBwZXIgc2FtcGxlIHNpemUiKQoKYGBgCgojIyBBbGwgZGF0YXNldHMKCmBgYHtyIGZpZy5oZWlnaHQ9MTAsIGZpZy53aWR0aD0xMn0KCiMgd3JhbmdsZQpzaW11bGF0aW9uIHw+CiAgdW5uZXN0KHJlc3VsdHMpIHw+CiAgbXV0YXRlKHNpZ25pZmljYW50ID0gcCA8IC4wNSwKICAgICAgICAgdG90YWxfbiA9IHBhc3RlKCJOID0iLCBuX3Blcl9jb25kaXRpb24qMiksCiAgICAgICAgIHRvdGFsX24gPSBmY3RfcmVvcmRlcih0b3RhbF9uLCBhcy5udW1lcmljKHN0cl9leHRyYWN0KHRvdGFsX24sICJcXGQrIikpKSkgfD4KICAjIHBsb3QKICBnZ3Bsb3QoYWVzKGl0ZXJhdGlvbiwgY29oZW5zX2QsIGNvbG9yID0gc2lnbmlmaWNhbnQpKSArCiAgZ2VvbV9wb2ludChhbHBoYSA9IDAuNSkgKwogIGdlb21fbGluZXJhbmdlKGFlcyh5bWluID0gY29oZW5zX2RfY2lfbG93ZXIsIHltYXggPSBjb2hlbnNfZF9jaV91cHBlciksIGFscGhhID0gMC4yKSArCiAgZ2VvbV9obGluZSh5aW50ZXJjZXB0ID0gMCwgbGluZXR5cGUgPSAiZG90dGVkIikgKwogIHNjYWxlX3hfY29udGludW91cyhicmVha3MgPSBicmVha3NfcHJldHR5KG4gPSAxMCksCiAgICAgICAgICAgICAgICAgICAgIG5hbWUgPSAiSXRlcmF0aW9uIikgKwogIHNjYWxlX3lfY29udGludW91cyhicmVha3MgPSBicmVha3NfcHJldHR5KG4gPSAxMCksCiAgICAgICAgICAgICAgICAgICAgICNsaW1pdHMgPSBjKDAsMSksCiAgICAgICAgICAgICAgICAgICAgIG5hbWUgPSAiQ29oZW4ncyBkIikgKwogIHRoZW1lX2xpbmVkcmF3KCkgKwogIHNjYWxlX2NvbG9yX3ZpcmlkaXNfZChiZWdpbiA9IDAuMywgZW5kID0gMC43KSArCiAgZmFjZXRfd3JhcCh+IHRvdGFsX24sIG5jb2wgPSAzLCBzY2FsZXMgPSAiZnJlZV95IikKCmBgYAoKIyMgT3JkZXJlZCBieSBDb2hlbidzIGQKClRoZSBhYm92ZSBwbG90IGlzIGhhcmQgdG8gdW5kZXJzdGFuZC4gTGV0J3Mgb3JkZXIgdGhlIENvaGVuJ3MgZHMgZnJvbSBzbWFsbGVzdCB0byBsYXJnZXN0LiAKCmBgYHtyIGZpZy5oZWlnaHQ9MTAsIGZpZy53aWR0aD0xMn0KCiMgd3JhbmdsZQpzaW11bGF0aW9uIHw+CiAgdW5uZXN0KHJlc3VsdHMpIHw+CiAgbXV0YXRlKHNpZ25pZmljYW50ID0gcCA8IC4wNSwKICAgICAgICAgdG90YWxfbiA9IHBhc3RlKCJOID0iLCBuX3Blcl9jb25kaXRpb24qMiksCiAgICAgICAgIHRvdGFsX24gPSBmY3RfcmVvcmRlcih0b3RhbF9uLCBhcy5udW1lcmljKHN0cl9leHRyYWN0KHRvdGFsX24sICJcXGQrIikpKSkgfD4KICBhcnJhbmdlKG5fcGVyX2NvbmRpdGlvbiwgY29oZW5zX2QpIHw+CiAgZ3JvdXBfYnkobl9wZXJfY29uZGl0aW9uKSB8PgogIG11dGF0ZShyYW5rID0gcm93X251bWJlcigpKSB8PgogIHVuZ3JvdXAoKSB8PgogICMgcGxvdAogIGdncGxvdChhZXMocmFuaywgY29oZW5zX2QsIGNvbG9yID0gc2lnbmlmaWNhbnQpKSArCiAgZ2VvbV9wb2ludCgpICsKICBnZW9tX2xpbmVyYW5nZShhZXMoeW1pbiA9IGNvaGVuc19kX2NpX2xvd2VyLCB5bWF4ID0gY29oZW5zX2RfY2lfdXBwZXIpLCBhbHBoYSA9IDAuMikgKwogIGdlb21faGxpbmUoeWludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gImRvdHRlZCIpICsKICBzY2FsZV94X2NvbnRpbnVvdXMoYnJlYWtzID0gYnJlYWtzX3ByZXR0eShuID0gMTApLAogICAgICAgICAgICAgICAgICAgICBuYW1lID0gIlJhbmtlZCBpdGVyYXRpb24iKSArCiAgc2NhbGVfeV9jb250aW51b3VzKGJyZWFrcyA9IGJyZWFrc19wcmV0dHkobiA9IDEwKSwKICAgICAgICAgICAgICAgICAgICAgI2xpbWl0cyA9IGMoMCwxKSwKICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJDb2hlbidzIGQiKSArCiAgdGhlbWVfbGluZWRyYXcoKSArCiAgc2NhbGVfY29sb3JfdmlyaWRpc19kKGJlZ2luID0gMC4zLCBlbmQgPSAwLjcpICsKICBmYWNldF93cmFwKH4gdG90YWxfbiwgbmNvbCA9IDMpCgpgYGAKCi0gTm90aWNlIHRoZSByZWxhdGlvbnNoaXAgYmV0d2VlbiA5NSUgQ0kgYW5kIHAgdmFsdWUgc2lnbmlmaWNhbmNlLgotIEEgY2VydGFpbiBwZXJjZW50YWdlIG9mIENvaGVuJ3MgZHMgYXJlIGdyZWVuIHZzLiBibHVlLiBXaGF0IGlzIHRoaXMgcGVyY2VudGFnZSBhbHNvIGNhbGxlZD8gV2hhdCBzdGF0aXN0aWNhbCBwcm9wZXJ0eT8KCiMgTWVhbiBDb2hlbidzIGQgYnkgc2FtcGxlIHNpemUKCk5vdyBsZXQncyBhdmVyYWdlIG92ZXIgdGhlIENvaGVuJ3MgZHMgaW4gZWFjaCBjb25kaXRpb24gdG8gZmluZCB0aGUgbWVhbiBDb2hlbidzIGQgYW5kIGl0cyBtZWFuIDk1JSBDSXMuCgpgYGB7ciBmaWcuaGVpZ2h0PTIuNSwgZmlnLndpZHRoPTZ9CgojIHdyYW5nbGUKc2ltdWxhdGlvbl9zdW1tYXJ5IDwtIHNpbXVsYXRpb24gfD4KICAjIHVubmVzdCByZXN1bHRzCiAgdW5uZXN0KHJlc3VsdHMpIHw+CiAgZ3JvdXBfYnkobl9wZXJfY29uZGl0aW9uKSB8PgogIHN1bW1hcml6ZShwcm9wb3J0aW9uX3NpZ25pZmljYW50ID0gbWVhbihwIDwgLjA1KSwgCiAgICAgICAgICAgIG1lYW5fY29oZW5zX2QgPSBtZWFuKGNvaGVuc19kKSwKICAgICAgICAgICAgbWVhbl9jb2hlbnNfZF9jaV9sb3dlciA9IG1lYW4oY29oZW5zX2RfY2lfbG93ZXIpLAogICAgICAgICAgICBtZWFuX2NvaGVuc19kX2NpX3VwcGVyID0gbWVhbihjb2hlbnNfZF9jaV91cHBlcikpIHw+CiAgbXV0YXRlKGNlbnRlcmVkX21lYW5fY29oZW5zX2RfY2lfbG93ZXIgPSBtZWFuX2NvaGVuc19kX2NpX2xvd2VyIC0gbWVhbl9jb2hlbnNfZCwKICAgICAgICAgY2VudGVyZWRfbWVhbl9jb2hlbnNfZF9jaV91cHBlciA9IG1lYW5fY29oZW5zX2RfY2lfdXBwZXIgLSBtZWFuX2NvaGVuc19kKQoKIyBwbG90IHJlc3VsdHMKcDEgPC0gZ2dwbG90KHNpbXVsYXRpb25fc3VtbWFyeSwgYWVzKG5fcGVyX2NvbmRpdGlvbioyLCBtZWFuX2NvaGVuc19kKSkgKwogIGdlb21fcG9pbnQoKSArCiAgZ2VvbV9saW5lcmFuZ2UoYWVzKHltaW4gPSBtZWFuX2NvaGVuc19kX2NpX2xvd2VyLCB5bWF4ID0gbWVhbl9jb2hlbnNfZF9jaV91cHBlcikpICsKICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQgPSAwLCBsaW5ldHlwZSA9ICJkb3R0ZWQiKSArCiAgc2NhbGVfeF9jb250aW51b3VzKGJyZWFrcyA9IGJyZWFrc19wcmV0dHkobiA9IDEwKSwKICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJUb3RhbCBzYW1wbGUgc2l6ZSIpICsKICBzY2FsZV95X2NvbnRpbnVvdXMoYnJlYWtzID0gYnJlYWtzX3ByZXR0eShuID0gNSksCiAgICAgICAgICAgICAgICAgICAgICNsaW1pdHMgPSBjKDAsMSksCiAgICAgICAgICAgICAgICAgICAgIG5hbWUgPSAiTWVhbiBDb2hlbidzIGRcbihhbmQgbWVhbiA5NSUgQ0lzKSIpICsKICB0aGVtZV9saW5lZHJhdygpICsKICBnZ3RpdGxlKCJQb3B1bGF0aW9uIENvaGVuJ3MgZCA9IDAuNSIpCgpwMQoKYGBgCgojIE1lYW4gQ29oZW4ncyBkIGFuZCBwb3dlciBieSBzYW1wbGUgc2l6ZQoKYGBge3IgZmlnLmhlaWdodD01LCBmaWcud2lkdGg9Nn0KCnAyIDwtIGdncGxvdChzaW11bGF0aW9uX3N1bW1hcnksIGFlcyhuX3Blcl9jb25kaXRpb24qMiwgcHJvcG9ydGlvbl9zaWduaWZpY2FudCkpICsKICBnZW9tX3BvaW50KCkgKwogIGdlb21faGxpbmUoeWludGVyY2VwdCA9IDAuMDUsIGxpbmV0eXBlID0gImRvdHRlZCIpICsKICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQgPSAwLjgwLCBsaW5ldHlwZSA9ICJkb3R0ZWQiKSArCiAgc2NhbGVfeF9jb250aW51b3VzKGJyZWFrcyA9IGJyZWFrc19wcmV0dHkobiA9IDEwKSwKICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJUb3RhbCBzYW1wbGUgc2l6ZSIpICsKICBzY2FsZV95X2NvbnRpbnVvdXMoYnJlYWtzID0gYnJlYWtzX3ByZXR0eShuID0gNSksCiAgICAgICAgICAgICAgICAgICAgIGxpbWl0cyA9IGMoMCwxKSwKICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJQcm9wb3J0aW9uIG9mIHNpZ25pZmljYW50XG5wLXZhbHVlcyIpICsKICB0aGVtZV9saW5lZHJhdygpIAoKcDEgKyBwMiArIHBsb3RfbGF5b3V0KG5jb2wgPSAxKQoKYGBgCgojIFNlc3Npb24gaW5mbwoKYGBge3J9CgpzZXNzaW9uSW5mbygpCgpgYGAKCgo=